#!/usr/bin/python
# coding: utf-8

#0,1,2
DEBUG_LEVEL = 2

import os
import re
import sys
from xml.dom.minidom import parse
from mensaje import mensaje
from collocations import tTestGetCollocations, tTestReplaceCollocations


import math
# Not defined in any module
def log2(val):
    return log(val,2)

# Subclass of dict.
class freq(dict) :
	def N(self) :
		n = 0
		for k,v in self.iteritems() :
			n += v
		return n
	def prob_key(self, key) :
		return 	float(self[key])/self.N()
	def entropy(self) :
		h = 0
		for x in self.keys() :
			p_x = self.prob_key(x)
			h += p_x * log2(p_x)
		return -h
	def topK(self, k) :
		l = self.items()
		l.sort(lambda x,y : cmp(x[1],y[1]), reverse=True)
		l = l[:k]
		return l

class freqNGram(freq) :
	def __init__(self,n) :
		self.ngram = n
	def add(self, grams) :
		for i in range(len(grams)-self.ngram) :
			g = tuple(grams[i:i+self.ngram+1]) # tuple is hashable
			#print "DEBUG:", g
			if self.has_key(g) :
				self[g] += 1
			else :
				self[g] = 1


# zipf law, crop the middle of the picture
def cropWordsZipf(corpus, lowPercent, highPercent, keepWords) :
        f0Gram = freqNGram(0) # freq of words, 0-grams
        f0Gram.add(corpus)
        # get freqs
        freqs = f0Gram.values()
        #print "DEBUG:", fNGram
        # frequencies of frequencies, using freqs as 0-gram
        ff0Gram = freqNGram(0)
        ff0Gram.add(freqs)
        #print "DEBUG: len(f0Gram), len(ff0Gram)", len(f0Gram), len(ff0Gram)
        l = ff0Gram.items()
        l.sort(lambda x,y : cmp(x[1],y[1]), reverse=True)
        #get min erasing first lowPercent frequencies, get max erasing highPercent frequencies
        min_freq = l[int(lowPercent*len(l))][0][0]
        max_freq = l[int(highPercent*len(l))][0][0]
        #print "DEBUG: min_freq, max_freq", min_freq, max_freq
        #print "DEBUG: before cutting len(corpus)", len(corpus)
        #for w in corpus :
        #    print w, f0Gram[tuple([w])], min_freq<=f0Gram[tuple([w])] and f0Gram[tuple([w])]<=max_freq
        corpus = filter((lambda w : (w in keepWords) or (min_freq<=f0Gram[tuple([w])] and f0Gram[tuple([w])]<=max_freq)), corpus)
        #print "DEBUG: after cutting len(corpus)", len(corpus)
        #print corpus
        return corpus



#
# Main
#

# It was 10
w_size = 4

# Notar que están incorporados las posibilidades luego de encontrar collocations
candidatos = {
        u'cristina': [u'cfk',u'kristina',u'kirchner',u'klishtina',u'cristina_kirchner', u'cristina_fernandez', u'cristina_fernández', u'néstor_kirchner', u'nestor_kirchner', u'los_k'],
        u'lavagna':[u'sr lavagna', u'roberto_lavagna', u'dr_lavagna'],
        u'murphy':[u'bulldog', u'buldog', u'murfy', u'lópez_murphy', u'lopez_murphy', u'bull_dog'],
        u'sobisch': [u'bigoton', u'bigotón'],
        u'ripoll' :[u'vilma_ripoll'],
        u'solanas':[u'pino', u'pinosolanas', u'pino_solanas'],
        u'carrio': [u'carrio', u'gorda', u'la_gorda', u'sra_carrió', u'elisa_carrio', u'elisa_carrió'],
        u'pitrola' :[],
        u'saa': [u'adolfo', u'rodriguez_saa', u'rodríguez_saa', u'r_saa'],
        u'pitrola' :[]
}

# generate candidatosForms from previous dict of lists
candidatosForms = {}
for canonical in candidatos.iterkeys() :
        candidatosForms[canonical] = canonical
        for alternative in candidatos[canonical] : 
                candidatosForms[alternative] = canonical

#print "DEBUG:", candidatosForms

# Load all corpus from candidates
corpus_candidate = {}
for candidato in candidatos.iterkeys():
    print "DEBUG: Loading", candidato
    try:
        fd = open("../corpus/paginas_institucionales/"+candidato+".txt", "r")	# open file
    except:
        continue
    corpus = [] # corpus is a list of words
    # All corpus are in latin-1
    corpus_candidate[candidato] = map(unicode.lower, re.findall(u'[a-zA-Z0-9áéíóúÁÉÍÓÚñÑüÜ]+',unicode(fd.read().decode('latin-1'))))
    fd.close()
    
# Load corpus
assert(len(sys.argv)>1)
assert(sys.argv[1])
print "DEBUG: Loading ", sys.argv[1]
fd = open(sys.argv[1], "r") # open file
corpus_forum = {} # corpus is a dictionary post_id : list of post words
m = mensaje()
doc = parse(fd)
for root in doc.childNodes :
	if root.localName=="corpus" :
		for message in root.childNodes :
			#print "DEBUG: ", message.localName
			if message.localName=="mensaje" :
				m.load_from_dom(message) # got the text, split into words
		                texto = re.sub("<br>|quot;|amp;|gt;|http://[^\s]*",'',m.texto)
                                ws = map(unicode.lower, re.findall(u'[a-zA-Z0-9áéíóúÁÉÍÓÚñÑüÜ]+',unicode(texto)))
				                #print "DEBUG:", ws
                                corpus_forum[m.id] = ws
                                #print corpus_forum[m.id]
fd.close()


# Accumulate all corpora to get collocations globally
all_corpus = []
for c in corpus_candidate.iterkeys() :
        all_corpus.extend(corpus_candidate[c])
for i in corpus_forum.iterkeys() :
        all_corpus.extend(corpus_forum[i])
# Generate collocations from all corpora. Computation INTENSIVE!!!
print "DEBUG: generating collocations"
collocations = tTestGetCollocations(all_corpus, topk=16384, topj=512)
#print "DEBUG: collocations", collocations
print "DEBUG:replacing collocations"
for c in corpus_candidate.iterkeys() :
        corpus_candidate[c] = tTestReplaceCollocations(corpus_candidate[c], collocations)
for i in corpus_forum.iterkeys() :
        corpus_forum[i] = tTestReplaceCollocations(corpus_forum[i], collocations)
all_corpus = [] #please GC all this


# Conflict with previous massage?

# Crop with the Zipf law, erase too common and too uncommon
for candidato in candidatos.iterkeys(): 
    # Erase unnecesary words from corpus
    old_len = len(corpus_candidate[candidato])
    corpus_candidate[candidato] = cropWordsZipf(corpus_candidate[candidato], 0.01, 0.9, candidatosForms.keys())
    new_len = len(corpus_candidate[candidato])
    print "DEBUG:", candidato, "len before, after cropping ", old_len, new_len
    

print "DEBUG: computing pos&neg words out of candidates corpora"
positive_words = {}
negative_words = {}
for canonical in candidatos.iterkeys():
    positive_words[canonical] = set()
    negative_words[canonical] = set()

for candidato in candidatos.iterkeys():
    #raw_input(candidato)
 
    corpus = corpus_candidate[candidato] # corpus is a list of words

    resultados = {}
    for k in candidatos.keys():
        resultados[k] = {}

    windows = {}

    for i in range(len(corpus)):
        if corpus[i] in candidatosForms.keys():
            start = max(0, i - w_size/2)
            end   = min(i + w_size/2 + 1, len(corpus))
            if corpus[i] in candidatosForms[candidato]: #es Lavagna en corpus de Lavagna: palabra positiva
                for j in range(start,end):
                    if corpus[j] not in candidatosForms.keys(): #no incluir nombres de candidatos ni sus apodos
                        positive_words[candidato].add(corpus[j])
            else: #es Lavagna en corpus de Carrió: palabra negativa
                for j in range(start,end):
                    if corpus[j] not in candidatosForms.keys(): #no incluir nombres de candidatos ni sus apodos
                        negative_words[candidatosForms[corpus[i]]].add(corpus[j])

# print pos&neg
for c in candidatos.iterkeys():
    print "DEBUG:", c, "+"+str(len(positive_words[c])), "-"+str(len(negative_words[c]))

#eliminar palabras que aparecen como positivas y negativas
print "DEBUG: erasing common pos&neg"
for c in candidatos.iterkeys():
    #print "palabras pos de ", c
    #print positive_words[c]
    #print "palabras neg de ", c
    #print negative_words[c]
    #print "palabras pos y neg de ", c
    intersection = negative_words[c].intersection(positive_words[c])
    #print intersection
    negative_words[c] = negative_words[c].difference(intersection)
    positive_words[c] = positive_words[c].difference(intersection)
    #raw_input("")
    #print "desp, palabras pos de ", c
    #print positive_words[c]
    #print "desp, palabras neg de ", c
    #print negative_words[c]
    #raw_input("")

# print pos&neg
for c in candidatos.iterkeys():
    print "DEBUG:", c, "+"+str(len(positive_words[c])), "-"+str(len(negative_words[c]))

"""
for c in candidatos.keys() :
    print "Candidato:", c
    if DEBUG_LEVEL > 1: raw_input("continuar")
    #neg = negative_words[c].items()
    #neg.sort(lambda x,y : cmp(x[1],y[1]), reverse=True)
    neg = negative_words[c]
    print "NEG:", neg
    pos = positive_words[c]
    print "POS:", pos
"""



#Analizar el corpus
        
# Window size to do the voting
w_size = 10

votacion = {} # cada candidato cuantos votos
for c in candidatos.iterkeys():
    votacion[c] = 0

for (post_id, post_words) in corpus_forum.iteritems() :
    
    result_positive = {}
    result_negative = {}
    #un solo entero que puede ser pos o neg
    result = {}
    for c in candidatos.iterkeys():
        result_positive[c] = 0
        result_negative[c] = 0
        result[c] = 0

    for i in range(len(post_words)) :
        
        if post_words[i] in candidatosForms.keys() :
            start = max(0, i - w_size/2)
            end   = min(i + w_size/2 + 1, len(post_words))
            #print "matchea= ",
            #print post_words[start:end]
            
            window = post_words[start:end]
            candidato = candidatosForms[post_words[i]]
            """                    
            for w in window:
                if w in positive_words[candidato] :
                    result_positive[candidato] += 1
                elif w in negative_words[candidato] :
                    for c in candidatos.iterkeys() :
                        if c != candidato : #las negativas suman como positivas para el resto
                            result_positive[c] += 1
                        else :
                            result_negative[c] += 1
            """
            for w in window :
                if w in positive_words[candidato] :
                    result[candidato] += 1
                elif w in negative_words[candidato] :
                    result[candidato] -= 1
    
    l = result.items()
    l.sort(lambda x,y : cmp(x[1],y[1]), reverse=True)
    #print "Post", post_id, "result", l
    if l[0][1]>0 and l[0][1]>l[1][1] : # el mejor es positivo y le gana al segundo
            votacion[l[0][0]] += 1
    
    """
    l = result_positive.items()
    l.sort(lambda x,y : cmp(x[1],y[1]), reverse=True)
    if DEBUG_LEVEL > 1:
        print "result_positive for post %s =" % (post_id)
        print l
    if l[0][1] > 2 * l[1][1]:
        print "post %s etiquetado como bueno para %s" %(post_id, l[0][0])

    l = result_negative.items()
    l.sort(lambda x,y : cmp(x[1],y[1]), reverse=True)
    if DEBUG_LEVEL > 1:
        print "result_negative for post %s =" % (post_id)
        print l
    if l[0][1] > 2 * l[1][1]:
        print "post %s etiquetado como malo para %s" %(post_id, l[0][0])
    """

print "Resultado de la votación"
l = votacion.items()
l.sort(lambda x,y : cmp(x[1],y[1]), reverse=True)
for c in l :
    print c[0], c[1]
print l[0][0], "presidente!"

exit(0)

if DEBUG_LEVEL > 0:
    inp = True
    while inp:
        inp = raw_input("Ver el contenido de algun post: ")
        if not inp:
            break
        texto = " ".join(corpus[inp])
        texto = re.sub("("+"|".join(candidatosForms.keys())+")",r"__\1__",texto)
        print(texto)
    



sys.exit(0)

